﻿using Newtonsoft.Json;
using SmartlForm.SimpleCrawler.Model;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using HtmlAgilityPack;
using System.Text.RegularExpressions;
using SmartlForm.Common;
using SmartlForm.Model;
using SmartlForm.DAL;

namespace SmartlForm.SimpleCrawler
{
    public class SmartCrawler
    {
        private static SimpleCrawler _crawler;

        /// <summary>
        /// url待抓队列 - 在同一调度时间中的待抓队列
        /// </summary>
        public static Dictionary<long, Queue<string>> urlQueue = new Dictionary<long, Queue<string>>();
        /// <summary>
        /// url抓取的历史记录  -  根据调度时间进行重置
        /// </summary>
        public static Dictionary<long, List<string>> urlHistory = new Dictionary<long, List<string>>();

        public static void Create(string url, SiteModel siteConfig)
        {
            if (_crawler == null)
            {
                _crawler = new SimpleCrawler();//调用爬虫程序
                _crawler.OnStart += (s, e) =>
                {
                    Console.WriteLine("爬虫开始抓取地址：" + e.Uri.ToString());
                };
                _crawler.OnError += (s, e) =>
                {
                    Console.WriteLine("爬虫抓取出现错误：" + e.Uri.ToString() + "，异常消息：" + e.Exception.ToString());
                };
                _crawler.OnCompleted += (s, e) =>
                {
                    var data = Crawling(e.PageSource, e.Uri.ToString(), e.SiteConfig);
                    if (data != null)
                    {
                        //抓取的数据
                        string result = JsonConvert.SerializeObject(data);
                        ArticleDao.Instance.Add(result, e.SiteConfig.Type, e.SiteConfig.accountId);

                        Console.WriteLine("===============================================");
                        Console.WriteLine("爬虫抓取任务完成！");
                        Console.WriteLine("耗时：" + e.Milliseconds + "毫秒");
                        Console.WriteLine("线程：" + e.ThreadId);
                        Console.WriteLine("地址：" + e.Uri.ToString());
                        Console.WriteLine("结果：" + result);
                    }
                    else
                    {
                        //无数据结果
                        Console.WriteLine("===============================================");
                        Console.WriteLine("暂未抓取到数据");
                        Console.WriteLine("耗时：" + e.Milliseconds + "毫秒");
                        Console.WriteLine("线程：" + e.ThreadId);
                        Console.WriteLine("地址：" + e.Uri.ToString());
                    }
                };
            }
            var logId = 1;
            _crawler.Start(new Uri(url), siteConfig, logId);
        }



        /// <summary>
        /// 爬取内容
        /// </summary>
        /// <returns></returns>
        private static object Crawling(string html, string url, SiteModel config)
        {
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);

            //解析多数据源规则
            if (config.Targets != null && config.Targets.Entitys != null)
            {
                if (config.Targets.SourceRules != null && config.Targets.SourceRules.Count > 0)
                {
                    foreach (var item in config.Targets.SourceRules)
                    {
                        if (item.Rule != null)
                        {
                            foreach (var rule in item.Rule)
                            {
                                if (rule.Field != null)
                                {
                                    foreach (var field in rule.Field)
                                    {
                                        Console.WriteLine("field.Parsers count：" + field.Parsers.Count);
                                        foreach (var parser in field.Parsers)
                                        {
                                            var pageNodes = doc.DocumentNode.SelectNodes(parser.XPath);
                                            if (pageNodes != null)
                                            {
                                                foreach (var pn in pageNodes)
                                                {
                                                    string sourceUrl = GetFieldValue(pn, field);
                                                    Console.WriteLine("来源地址：" + sourceUrl);



                                                    List<string> history;
                                                    if (urlHistory.ContainsKey(config.Id))
                                                    {
                                                        history = urlHistory[config.Id];
                                                        if (history == null)
                                                        {
                                                            history = new List<string>();
                                                        }
                                                    }
                                                    else
                                                    {
                                                        history = new List<string>();
                                                    }

                                                    //添加抓取目标
                                                    Queue<string> queue;
                                                    if (urlQueue.ContainsKey(config.Id))
                                                    {
                                                        queue = urlQueue[config.Id];
                                                    }
                                                    else
                                                    {
                                                        queue = new Queue<string>();
                                                    }
                                                    //判断url是否存在在队列中
                                                    if (!history.Contains(sourceUrl))
                                                    {
                                                        queue.Enqueue(sourceUrl);
                                                    }
                                                    else
                                                    {
                                                        Console.WriteLine("url queue 已存在：" + sourceUrl);
                                                    }
                                                    urlQueue[config.Id] = queue;
                                                    Console.WriteLine("url queue count：" + queue.Count.ToString());

                                                    //添加历史记录
                                                    history.Add(sourceUrl);
                                                    urlHistory[config.Id] = history;
                                                }
                                            }
                                        }


                                    }
                                }
                            }
                        }
                    }
                }
                return GetEntitys(doc, url, config);
            }
            return null;
        }

        /// <summary>
        /// 获取实体信息
        /// </summary>
        /// <param name="doc"></param>
        /// <returns></returns>
        private static object GetEntitys(HtmlDocument doc, string url, SiteModel config)
        {
            foreach (var item in config.Targets.Entitys)
            {
                //如果配置了实体模型
                if (item != null)
                {
                    //抓取主体信息
                    Dictionary<string, object> mainEntity = null;
                    if (item.Field != null && item.Field.Count > 0)
                    {
                        mainEntity = new Dictionary<string, object>();
                        foreach (var field in item.Field)
                        {
                            //从当前节点向下寻找（xpath需以.开头）
                            var currentNode = doc.DocumentNode.SelectSingleNode(field.Parsers[0].XPath);
                            //获取节点的值
                            string value = GetFieldValue(currentNode, field);
                            mainEntity.Add(field.Name, value);
                        }
                    }

                    object result;

                    //抓取集合信息
                    List<Dictionary<string, object>> arrayEntity = null;
                    //如果存在集合节点
                    if (item.Array != null && item.Array.Field.Count > 0 && !string.IsNullOrEmpty(item.Array.XPath))
                    {
                        arrayEntity = new List<Dictionary<string, object>>();
                        //查找循环节点
                        var nodes = doc.DocumentNode.SelectNodes(item.Array.XPath);
                        if (nodes == null)
                            continue;
                        //遍历循环节点
                        foreach (var node in nodes)
                        {
                            Dictionary<string, object> entity = new Dictionary<string, object>();
                            foreach (var field in item.Array.Field)
                            {
                                //从当前节点向下寻找（xpath需以.开头）
                                var currentNode = node.SelectSingleNode(field.Parsers[0].XPath);
                                string value = GetFieldValue(currentNode, field);
                                entity.Add(field.Name, value);
                            }
                            //抓取数据量+1
                            //FormBrowser.DataCount++;

                            arrayEntity.Add(entity);
                        }
                    }
                    //如果只有数组信息
                    if (mainEntity == null && arrayEntity != null)
                    {
                        result = arrayEntity;
                    }
                    else if (mainEntity != null && arrayEntity != null)
                    {
                        mainEntity.Add(item.Array.Name, arrayEntity);
                        result = mainEntity;
                    }
                    else
                    {
                        result = mainEntity;
                    }
                    return result;
                }
            }
            return null;
        }


        /// <summary>
        /// 获取字段值
        /// </summary>
        /// <param name="node"></param>
        /// <param name="field"></param>
        /// <returns></returns>
        private static string GetFieldValue(HtmlNode node, FieldModel field)
        {
            string value = string.Empty;
            if (node != null)
            {
                //如果需要获取属性值
                if (!string.IsNullOrEmpty(field.Parsers[0].Attribute))
                {
                    value = node.Attributes[field.Parsers[0].Attribute].Value;
                }
                else
                {
                    value = node.InnerText;
                }
                //正则匹配
                if (!string.IsNullOrEmpty(field.Parsers[0].Exp))
                {
                    if (string.IsNullOrEmpty(field.Parsers[0].ExpIndex))
                        value = new Regex(field.Parsers[0].Exp).Match(value).Value;
                    else
                    {
                        //索引从0开始
                        int index = field.Parsers[0].ExpIndex.ToInt32() + 1;
                        value = new Regex(field.Parsers[0].Exp).Match(value).Groups[index].Value;
                    }
                }
                //格式化
                if (!string.IsNullOrEmpty(field.Parsers[0].Format))
                {
                    value = field.Parsers[0].Format.Replace("$this", value);
                }
            }

            //去空格
            if (field.IsTrim)
            {
                value = value.Trim();
            }
            //删除html
            if (field.IsDelHtml)
            {
                value = StrHelper.RemoveHTML(value);
            }
            return value;
        }

    }
}
